package lucene;

import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;

import lucene.analyzer.PorterStemAnalyzer;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.snowball.SnowballAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.Version;
import org.tartarus.snowball.ext.EnglishStemmer;

public class AnalyzerDemo {
	public static void main(String[] args) throws IOException {
		// String s =
		// "He's Juexian, His email is forfuture1978@gmail.com. He's an ip address 192.168.0.1,AT&T and I.B.M. are all great companies.";
		String s = "My friends Jerry went to school to see his students but found them drunk which is not allowed.";
		
		tokenize(s, new StandardAnalyzer(Version.LUCENE_30));
		tokenize(s, new PorterStemAnalyzer());
		tokenize(s, new SnowballAnalyzer(Version.LUCENE_30, "English"));
		tokenize(s, new KeywordAnalyzer());
		tokenize(s, new SimpleAnalyzer());
		tokenize(s, new StopAnalyzer(Version.LUCENE_30));
		tokenize(s, new WhitespaceAnalyzer());
	}
	
	private static void tokenize(String s, Analyzer analyzer) throws IOException {
		System.out.println(analyzer.getClass().getName());
		StringReader reader = new StringReader(s);
		TokenStream ts = analyzer.tokenStream("field", reader);
		boolean hasnext = ts.incrementToken();
		while (hasnext) {
			TermAttribute ta = ts.getAttribute(TermAttribute.class);
			System.out.print(ta.term() + " | ");
			hasnext = ts.incrementToken();
		}
		System.out.println();
	}

}
